#!/bin/bash

#
# GuangZhou - doc file, need download!
#
# HunanProvince +1, Good!
#

city='佛山市'
base_url="http://www.foshan.gov.cn/gzjg/stjj/ztzl_1110965/tjgb_1110961/index.html"
list_url=$base_url""
abs_path=y
doc_file=n
pdf_file=y
get_html=y

year=$1

if [ -z $year ];then
	echo "usage: $0 <year>"
	echo "example: $0 2021"
	exit 1
fi

file_name="${year}年${city}国民经济和社会发展统计公报"

if [ "$abs_path" == "y" ];then
	year_uri=`curl -s -L $list_url | grep -E "${year}年${city}国民经济和社会发展统计公报|${year}年.*经济和社会发展概况" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed 's/"//g'`
	#year_uri=`curl -s -L $list_url | grep -E "${year}年${city}国民经济和社会发展统计公报|${year}年.*经济和社会发展概况" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed "s/'//g"`
else
	#year_uri=`curl -s -L $list_url | grep -E "${year}年.*国民经济和社会发展统计公报|${year}年.*经济和社会发展概况" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed 's/"//g' | sed 's/\.//'`
	year_uri=`curl -s -L $list_url | grep -E "${year}年.*国民经济和社会发展统计公报|${year}年.*经济和社会发展概况" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed 's/"//g'`
	#year_uri=`curl -s -L $list_url | grep -o -P "<a href=\"/htm/detail.*?title=\"${year}年${city}国民经济和社会发展统计公报\">" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed 's/"//g'`
fi

#echo "$year uri: $year_uri"
#exit 0

if [ -z "$year_uri" ];then
	echo "err: parse year uri failed!"
	exit 2
fi

year_uri=`echo $year_uri|awk '{print $1}'`

if [ ${year_uri:0:4} == "http" ];then
	year_url=${year_uri}
else
	year_url="${base_url}"${year_uri}
fi

echo "$year URL: $year_url"

if [ "${year_url:0-3:3}" == "pdf" ];then
	echo "PDF Url: $year_url"
	curl -sL $year_url -o "${file_name}.pdf"
	pdftotext ${file_name}.pdf
	if [ $? -ne 0 ];then
		echo "exec pdftotext ${file_name}.pdf failed!"
		exit 0
	fi
	get_html=n
	doc_file=n
	pdf_file=n
fi

if [ "$get_html" == "y" ];then
	report="`curl -s -L $year_url`"
	curl -s -L $year_url -o tmp.html
fi

if [ "$doc_file" == "y" ];then
	#doc_url=`echo $report | grep -E "${year}年.*国民经济和社会发展统计公报.doc" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed 's/"//g'`
	doc_url=`cat tmp.html | grep -E "${year}年.*国民经济和社会发展统计公报.doc" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed 's/"//g'`
	echo "Doc Url: $doc_url"
	curl -sL $doc_url -o "${file_name}.doc"
	exit 0
fi

if [ "$pdf_file" == "y" ];then
	#pdf_uri=`echo $report | grep -E "${year}年.*国民经济和社会发展统计公报.pdf" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed 's/"//g'`
	#pdf_uri=`cat tmp.html | grep -E "${year}年.*国民经济和社会发展统计公报.pdf" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed 's/"//g'`
	pdf_uri=`cat tmp.html | grep -E "${year}年.*经济和社会发展统计公报.pdf" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed 's/"//g'`
	echo "PDF Uri: $pdf_uri"
	if [ ${pdf_uri:0:4} == "http" ];then
		pdf_url=${pdf_uri}
	else
		pdf_url="${base_url}"${pdf_uri}
	fi
	curl -sL $pdf_url -o "${file_name}.pdf"
	pdftotext ${file_name}.pdf
	if [ $? -ne 0 ];then
		echo "exec pdftotext ${file_name}.pdf failed!"
		exit 0
	fi
fi

if [ -f ${file_name}.txt ];then
	sed -i ':a;N;s/\n//g;ta' ${file_name}.txt
	report="`cat ${file_name}.txt`"
else
	exit 0
fi

# dict map
declare -A fields #(["常住人口"]=0, ["生产总值"]=0, ["一般公共预算收入"]=0, ["一般公共预算支出"]=0)

# for sequence
key_list=("常住人口" "全市地区生产总值" "一般公共预算收入" "一般公共预算支出")
#key_list=("全市常住人口" "实现地区生产总值" "一般公共预算收入" "一般公共预算支出")

#for key in ${!fields[*]};
for key in ${key_list[*]};
do
	fields[$key]=`echo $report | awk -F "$key" '{ print $2 }' | awk -F '，' '{ print $1 }' | perl -pe 's/<.*?>//g' | sed 's/\[.*\]//g'`
	#fields[$key]=`cat tmp.html | awk -F "$key" '{ print $2 }' | awk -F '，' '{ print $1 }' | perl -pe 's/<.*?>//g' | sed 's/\[.*\]//g'`
	echo "$key: ${fields[$key]}"
done

exit 0

